import sys
sys.path.append("..")
from database.db_handler import MysqlHander
from common.my_http import MyHttp
import urllib.request
from bs4 import BeautifulSoup
from common.entity_data import EntityData
import glob
import jieba
import json
import re
import threading
import hashlib

'''
品牌
'''

class Quchong:
    def __init__(self):
        self.entity = EntityData()
        self.data_map = {}
        pass
    
    def process(self):
        id = 0
        while True:
            datas = self.entity.query_entity_for_index(id)
            if len(datas) == 0:
                break
            for data in datas:
                data_id = data[0]
                data_split = [x for x in data[2].split('|') if x]
                new_data_arr = list(set(data_split))
                new_data_arr.sort(key = data_split.index)
                new_data = "|".join(new_data_arr)
                print(new_data)
                self.entity.update_set_by_id(data_id, new_data)
                id = data_id


def quchong_api():
    d = Quchong()
    d.process()

if __name__ == '__main__':
    quchong_api()
